

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">


<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    
    <title>contributed.parallel_map &mdash; agpy 0.1.2 documentation</title>
    
    <link rel="stylesheet" href="../../_static/extra.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
    
    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '../../',
        VERSION:     '0.1.2',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  true
      };
    </script>
    <script type="text/javascript" src="../../_static/jquery.js"></script>
    <script type="text/javascript" src="../../_static/underscore.js"></script>
    <script type="text/javascript" src="../../_static/doctools.js"></script>
    <link rel="top" title="agpy 0.1.2 documentation" href="../../index.html" />
    <link rel="up" title="Module code" href="../index.html" />
     
    <script type="text/javascript">

      var _gaq = _gaq || [];
      _gaq.push(['_setDomainName', 'pyspeckit.bitbucket.org']);
      _gaq.push(['_setAllowHash', false]);
      _gaq.push(['_trackPageview']);


    </script>
    <link rel="stylesheet" type="text/css" href="../../_static/extra.css" />
  </head>
  <body>
    <div class="header-wrapper">
      <div class="header">
        <h1><a href="../../index.html">agpy 0.1.2 documentation</a></h1>
        <div class="rel">
          <a href="http://agpy.googlecode.com">agpy Home </a>  |
          <a href=../../index.html>Docs Home </a>  |
          <a href="http://code.google.com/p/agpy/w/list">Wiki</a>  |
          <a href=../../search.html>Search </a>
        </div>
       </div>
    </div>

    <div class="content-wrapper">
      <div class="content">
        <div class="document">
            
      <div class="documentwrapper">
        <div class="bodywrapper">
          <div class="body">
            
  <h1>Source code for contributed.parallel_map</h1><div class="highlight"><pre>
<span class="kn">import</span> <span class="nn">numpy</span>
<span class="n">_multi</span><span class="o">=</span><span class="bp">False</span>
<span class="n">_ncpus</span><span class="o">=</span><span class="mi">1</span>

<span class="k">try</span><span class="p">:</span>
  <span class="c"># May raise ImportError</span>
  <span class="kn">import</span> <span class="nn">multiprocessing</span>
  <span class="n">_multi</span><span class="o">=</span><span class="bp">True</span>

  <span class="c"># May raise NotImplementedError</span>
  <span class="n">_ncpus</span> <span class="o">=</span> <span class="n">multiprocessing</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
<span class="k">except</span><span class="p">:</span>
  <span class="k">pass</span>


<span class="n">__all__</span> <span class="o">=</span> <span class="p">(</span><span class="s">&#39;parallel_map&#39;</span><span class="p">,)</span>


<span class="k">def</span> <span class="nf">worker</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">ii</span><span class="p">,</span> <span class="n">chunk</span><span class="p">,</span> <span class="n">out_q</span><span class="p">,</span> <span class="n">err_q</span><span class="p">,</span> <span class="n">lock</span><span class="p">):</span>
  <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">  A worker function that maps an input function over a</span>
<span class="sd">  slice of the input iterable.</span>

<span class="sd">  :param f  : callable function that accepts argument from iterable</span>
<span class="sd">  :param ii  : process ID</span>
<span class="sd">  :param chunk: slice of input iterable</span>
<span class="sd">  :param out_q: thread-safe output queue</span>
<span class="sd">  :param err_q: thread-safe queue to populate on exception</span>
<span class="sd">  :param lock : thread-safe lock to protect a resource</span>
<span class="sd">         ( useful in extending parallel_map() )</span>
<span class="sd">  &quot;&quot;&quot;</span>
  <span class="n">vals</span> <span class="o">=</span> <span class="p">[]</span>

  <span class="c"># iterate over slice </span>
  <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">chunk</span><span class="p">:</span>
    <span class="k">try</span><span class="p">:</span>
      <span class="n">result</span> <span class="o">=</span> <span class="n">f</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
    <span class="k">except</span> <span class="ne">Exception</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span>
      <span class="n">err_q</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
      <span class="k">return</span>

    <span class="n">vals</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>

  <span class="c"># output the result and task ID to output queue</span>
  <span class="n">out_q</span><span class="o">.</span><span class="n">put</span><span class="p">(</span> <span class="p">(</span><span class="n">ii</span><span class="p">,</span> <span class="n">vals</span><span class="p">)</span> <span class="p">)</span>


<span class="k">def</span> <span class="nf">run_tasks</span><span class="p">(</span><span class="n">procs</span><span class="p">,</span> <span class="n">err_q</span><span class="p">,</span> <span class="n">out_q</span><span class="p">,</span> <span class="n">num</span><span class="p">):</span>
  <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">  A function that executes populated processes and processes</span>
<span class="sd">  the resultant array. Checks error queue for any exceptions.</span>

<span class="sd">  :param procs: list of Process objects</span>
<span class="sd">  :param out_q: thread-safe output queue</span>
<span class="sd">  :param err_q: thread-safe queue to populate on exception</span>
<span class="sd">  :param num : length of resultant array</span>

<span class="sd">  &quot;&quot;&quot;</span>
  <span class="c"># function to terminate processes that are still running.</span>
  <span class="n">die</span> <span class="o">=</span> <span class="p">(</span><span class="k">lambda</span> <span class="n">vals</span> <span class="p">:</span> <span class="p">[</span><span class="n">val</span><span class="o">.</span><span class="n">terminate</span><span class="p">()</span> <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">vals</span>
             <span class="k">if</span> <span class="n">val</span><span class="o">.</span><span class="n">exitcode</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">])</span>

  <span class="k">try</span><span class="p">:</span>
    <span class="k">for</span> <span class="n">proc</span> <span class="ow">in</span> <span class="n">procs</span><span class="p">:</span>
      <span class="n">proc</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>

    <span class="k">for</span> <span class="n">proc</span> <span class="ow">in</span> <span class="n">procs</span><span class="p">:</span>
      <span class="n">proc</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>

  <span class="k">except</span> <span class="ne">Exception</span><span class="p">,</span> <span class="n">e</span><span class="p">:</span>
    <span class="c"># kill all slave processes on ctrl-C</span>
    <span class="n">die</span><span class="p">(</span><span class="n">procs</span><span class="p">)</span>
    <span class="k">raise</span> <span class="n">e</span>

  <span class="k">if</span> <span class="ow">not</span> <span class="n">err_q</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
    <span class="c"># kill all on any exception from any one slave</span>
    <span class="n">die</span><span class="p">(</span><span class="n">procs</span><span class="p">)</span>
    <span class="k">raise</span> <span class="n">err_q</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>

  <span class="c"># Processes finish in arbitrary order. Process IDs double</span>
  <span class="c"># as index in the resultant array.</span>
  <span class="n">results</span><span class="o">=</span><span class="p">[</span><span class="bp">None</span><span class="p">]</span><span class="o">*</span><span class="n">num</span><span class="p">;</span>
  <span class="k">while</span> <span class="ow">not</span> <span class="n">out_q</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
    <span class="n">idx</span><span class="p">,</span> <span class="n">result</span> <span class="o">=</span> <span class="n">out_q</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
    <span class="n">results</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span>

  <span class="k">try</span><span class="p">:</span>
      <span class="c"># Remove extra dimension added by array_split</span>
      <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="n">numpy</span><span class="o">.</span><span class="n">concatenate</span><span class="p">(</span><span class="n">results</span><span class="p">))</span>
  <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
      <span class="k">return</span> <span class="n">results</span>


<div class="viewcode-block" id="parallel_map"><a class="viewcode-back" href="../../contributed.html#contributed.parallel_map.parallel_map">[docs]</a><span class="k">def</span> <span class="nf">parallel_map</span><span class="p">(</span><span class="n">function</span><span class="p">,</span> <span class="n">sequence</span><span class="p">,</span> <span class="n">numcores</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
  <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">  A parallelized version of the native Python map function that</span>
<span class="sd">  utilizes the Python multiprocessing module to divide and </span>
<span class="sd">  conquer sequence.</span>

<span class="sd">  parallel_map does not yet support multiple argument sequences.</span>

<span class="sd">  :param function: callable function that accepts argument from iterable</span>
<span class="sd">  :param sequence: iterable sequence </span>
<span class="sd">  :param numcores: number of cores to use</span>
<span class="sd">  &quot;&quot;&quot;</span>
  <span class="k">if</span> <span class="ow">not</span> <span class="nb">callable</span><span class="p">(</span><span class="n">function</span><span class="p">):</span>
    <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">&quot;input function &#39;</span><span class="si">%s</span><span class="s">&#39; is not callable&quot;</span> <span class="o">%</span>
              <span class="nb">repr</span><span class="p">(</span><span class="n">function</span><span class="p">))</span>

  <span class="k">if</span> <span class="ow">not</span> <span class="n">numpy</span><span class="o">.</span><span class="n">iterable</span><span class="p">(</span><span class="n">sequence</span><span class="p">):</span>
    <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">&quot;input &#39;</span><span class="si">%s</span><span class="s">&#39; is not iterable&quot;</span> <span class="o">%</span>
              <span class="nb">repr</span><span class="p">(</span><span class="n">sequence</span><span class="p">))</span>

  <span class="n">size</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">sequence</span><span class="p">)</span>

  <span class="k">if</span> <span class="ow">not</span> <span class="n">_multi</span> <span class="ow">or</span> <span class="n">size</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
    <span class="k">return</span> <span class="nb">map</span><span class="p">(</span><span class="n">function</span><span class="p">,</span> <span class="n">sequence</span><span class="p">)</span>

  <span class="k">if</span> <span class="n">numcores</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
    <span class="n">numcores</span> <span class="o">=</span> <span class="n">_ncpus</span>

  <span class="c"># Returns a started SyncManager object which can be used for sharing </span>
  <span class="c"># objects between processes. The returned manager object corresponds</span>
  <span class="c"># to a spawned child process and has methods which will create shared</span>
  <span class="c"># objects and return corresponding proxies.</span>
  <span class="n">manager</span> <span class="o">=</span> <span class="n">multiprocessing</span><span class="o">.</span><span class="n">Manager</span><span class="p">()</span>

  <span class="c"># Create FIFO queue and lock shared objects and return proxies to them.</span>
  <span class="c"># The managers handles a server process that manages shared objects that</span>
  <span class="c"># each slave process has access to. Bottom line -- thread-safe.</span>
  <span class="n">out_q</span> <span class="o">=</span> <span class="n">manager</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
  <span class="n">err_q</span> <span class="o">=</span> <span class="n">manager</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
  <span class="n">lock</span> <span class="o">=</span> <span class="n">manager</span><span class="o">.</span><span class="n">Lock</span><span class="p">()</span>

  <span class="c"># if sequence is less than numcores, only use len sequence number of </span>
  <span class="c"># processes</span>
  <span class="k">if</span> <span class="n">size</span> <span class="o">&lt;</span> <span class="n">numcores</span><span class="p">:</span>
    <span class="n">numcores</span> <span class="o">=</span> <span class="n">size</span> 

  <span class="c"># group sequence into numcores-worth of chunks</span>
  <span class="n">sequence</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">array_split</span><span class="p">(</span><span class="n">sequence</span><span class="p">,</span> <span class="n">numcores</span><span class="p">)</span>

  <span class="n">procs</span> <span class="o">=</span> <span class="p">[</span><span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">worker</span><span class="p">,</span>
           <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">function</span><span class="p">,</span> <span class="n">ii</span><span class="p">,</span> <span class="n">chunk</span><span class="p">,</span> <span class="n">out_q</span><span class="p">,</span> <span class="n">err_q</span><span class="p">,</span> <span class="n">lock</span><span class="p">))</span>
         <span class="k">for</span> <span class="n">ii</span><span class="p">,</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">sequence</span><span class="p">)]</span>

  <span class="k">return</span> <span class="n">run_tasks</span><span class="p">(</span><span class="n">procs</span><span class="p">,</span> <span class="n">err_q</span><span class="p">,</span> <span class="n">out_q</span><span class="p">,</span> <span class="n">numcores</span><span class="p">)</span>

</div>
<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s">&quot;__main__&quot;</span><span class="p">:</span>
  <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">  Unit test of parallel_map()</span>

<span class="sd">  Create an arbitrary length list of references to a single</span>
<span class="sd">  matrix containing random floats and compute the eigenvals</span>
<span class="sd">  in serial and parallel. Compare the results and timings.</span>
<span class="sd">  &quot;&quot;&quot;</span>

  <span class="kn">import</span> <span class="nn">time</span>

  <span class="n">numtasks</span> <span class="o">=</span> <span class="mi">5</span>
  <span class="c">#size = (1024,1024)</span>
  <span class="n">size</span> <span class="o">=</span> <span class="p">(</span><span class="mi">512</span><span class="p">,</span><span class="mi">512</span><span class="p">)</span>

  <span class="n">vals</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="o">*</span><span class="n">size</span><span class="p">)</span>
  <span class="n">f</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">eigvals</span>

  <span class="n">iterable</span> <span class="o">=</span> <span class="p">[</span><span class="n">vals</span><span class="p">]</span><span class="o">*</span><span class="n">numtasks</span>

  <span class="k">print</span> <span class="p">(</span><span class="s">&#39;Running numpy.linalg.eigvals </span><span class="si">%i</span><span class="s">X on matrix size [</span><span class="si">%i</span><span class="s">,</span><span class="si">%i</span><span class="s">]&#39;</span> <span class="o">%</span>
      <span class="p">(</span><span class="n">numtasks</span><span class="p">,</span><span class="n">size</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span><span class="n">size</span><span class="p">[</span><span class="mi">1</span><span class="p">]))</span>

  <span class="n">tt</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
  <span class="n">presult</span> <span class="o">=</span> <span class="n">parallel_map</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">iterable</span><span class="p">)</span>
  <span class="k">print</span> <span class="s">&#39;parallel map in </span><span class="si">%g</span><span class="s"> secs&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span><span class="o">-</span><span class="n">tt</span><span class="p">)</span>

  <span class="n">tt</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
  <span class="n">result</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">iterable</span><span class="p">)</span>
  <span class="k">print</span> <span class="s">&#39;serial map in </span><span class="si">%g</span><span class="s"> secs&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span><span class="o">-</span><span class="n">tt</span><span class="p">)</span>

  <span class="k">assert</span> <span class="p">(</span><span class="n">numpy</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">==</span> <span class="n">numpy</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">presult</span><span class="p">))</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
</pre></div>

          </div>
        </div>
      </div>
        </div>
        <div class="sidebar">
          <h3>Table Of Contents</h3>
          <ul>
<li class="toctree-l1"><a class="reference internal" href="../../agpy.html">Adam Ginsburg&#8217;s Python Code (agpy)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../image_tools.html">Image Tools</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../fft_tools.html">AG_fft_tools Package</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../plfit.html">plfit Package</a></li>
</ul>

          <h3 style="margin-top: 1.5em;">Search</h3>
          <form class="search" action="../../search.html" method="get">
            <input type="text" name="q" />
            <input type="submit" value="Go" />
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>
          <p class="searchtip" style="font-size: 90%">
            Enter search terms or a module, class or function name.
          </p>
        </div>
        <div class="clearer"></div>
      </div>
    </div>

    <div class="footer">
      &copy; Copyright 2011, Adam Ginsburg.
      Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.2pre.
    <script type="text/javascript">

      var _gaq = _gaq || [];
      _gaq.push(['_setAccount', 'UA-6253248-2']);
      _gaq.push(['_trackPageview']);

      (function() {
        var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
        ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
        var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
      })();

    </script>
        
    </div>
  </body>
</html>